import time
import requests
import json
import csv

artice_href='https://xuangubao.cn/article/'
theme_href='https://xuangubao.cn/theme/'

file = open("xuangu.csv", 'wt', newline='', encoding='utf-8')
writer = csv.writer(file)
writer.writerow(('文章的id', '标题', '详细内容', '凑字段', '股票链接', '股票名称','不知名凑字段id'))

def get_url(url):
    headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'}
    res = requests.get(url, headers=headers)
    return res


def get_all(res):
    all = json.loads(res.text)                          #把json格式进行解析为python所认识的
    next_cursor = (all['data']['next_cursor'])           #网址之间不一样的数据
    a=len(all['data']['messages'])                        #获取messages的长度
    for i in range(0,a):
        zhu_id=(all['data']['messages'][i]['id'])           #文章的id
        title = (all['data']['messages'][i]['title'])        #标题
        summary= (all['data']['messages'][i]['summary'])     #详细内容
        subj_ids=(all['data']['messages'][i]['subj_ids'])    #凑字段
        b = len(all['data']['messages'][i]['bkj_infos'])     #获取messages里面的bkj_infos的长度
        fu_id=[]
        name=[]
        for j in range(0, b):
            fus_id=(all['data']['messages'][i]['bkj_infos'][j]['id'])     #文章所属股票id
            app = theme_href + str(fus_id)                               #将所得id与股票部分链接进行连接
            fu_id.append(app)                                           #加入元素
            names = (all['data']['messages'][i]['bkj_infos'][j]['name'])   #文章所属股票名称
            name.append(names)
        manual_updated_at = (all['data']['messages'][i]['manual_updated_at'])  #不知名凑字段id
      #  print(artice_href+str(zhu_id),title,summary, subj_ids,str(fu_id),name)
        writer.writerow((artice_href+str(zhu_id),title,summary, subj_ids,str(fu_id),name,manual_updated_at))
    next_url=get_nexturl(next_cursor)      #调用函数获取下一个网页的网址
    return  next_url


def get_nexturl(next_cursor):
    next_url = 'https://baoer-api.xuangubao.cn/api/v6/message/newsflash?limit=20&cursor={}&subj_ids=9,10,723,35,469,821&platform=pcweb'.format(next_cursor)
    return next_url

if __name__ == '__main__':
    url = 'https://baoer-api.xuangubao.cn/api/v6/message/newsflash?limit=20&subj_ids=9,10,723,35,469,821&platform=pcweb'
    for i in range(1,1000):
        res=get_url(url)
        next_url=get_all(res)
        print('已完成第{}页面'.format(i))
        time.sleep(1)

